From 01170a1cfefdc46a8d832fcbbd745ba324696651 Mon Sep 17 00:00:00 2001 From: "mafetter@fleming.research" Date: Thu, 24 Mar 2005 16:48:36 +0000 Subject: [PATCH] bitkeeper revision 1.1261 (4242ef64dXDbGRaZN94_Vg02rxL1tg) Snapshots of L1 page table pages now only snapshot the active portion of the page. Improved the tlb flushing of shadow mode somewhat... Fixed a bug in the shadow_min_max encoding stuff. Signed-off-by: michael.fetterman@cl.cam.ac.uk --- xen/arch/x86/shadow.c | 88 ++++++++++++++++++++++++++++-------- xen/include/asm-x86/shadow.h | 15 ++++-- xen/include/xen/perfc_defn.h | 35 +++++++++----- 3 files changed, 104 insertions(+), 34 deletions(-) diff --git a/xen/arch/x86/shadow.c b/xen/arch/x86/shadow.c index 12c4683a19..900dd910df 100644 --- a/xen/arch/x86/shadow.c +++ b/xen/arch/x86/shadow.c @@ -124,7 +124,7 @@ shadow_demote(struct domain *d, unsigned long gpfn, unsigned long gmfn) { ASSERT(frame_table[gmfn].count_info & PGC_page_table); - if ( shadow_max_pgtable_type(d, gpfn) == PGT_none ) + if ( shadow_max_pgtable_type(d, gpfn, NULL) == PGT_none ) { clear_bit(_PGC_page_table, &frame_table[gmfn].count_info); @@ -274,6 +274,10 @@ alloc_shadow_page(struct domain *d, break; } + // Don't add a new shadow of something that already has a snapshot. + // + ASSERT( (psh_type == PGT_snapshot) || !mfn_out_of_sync(gmfn) ); + set_shadow_status(d, gpfn, gmfn, smfn, psh_type); if ( pin ) @@ -1566,8 +1570,10 @@ static inline unsigned long shadow_make_snapshot( struct domain *d, unsigned long gpfn, unsigned long gmfn) { - unsigned long smfn; + unsigned long smfn, sl1mfn; void *original, *snapshot; + u32 min_max = 0; + int min, max, length; if ( test_and_set_bit(_PGC_out_of_sync, &frame_table[gmfn].count_info) ) { @@ -1588,9 +1594,21 @@ shadow_make_snapshot( if ( !get_shadow_ref(smfn) ) BUG(); + if ( shadow_max_pgtable_type(d, gpfn, &sl1mfn) == PGT_l1_shadow ) + min_max = pfn_to_page(sl1mfn)->tlbflush_timestamp; + pfn_to_page(smfn)->tlbflush_timestamp = min_max; + + min = SHADOW_MIN(min_max); + max = SHADOW_MAX(min_max); + length = max - min + 1; + perfc_incr_histo(snapshot_copies, length, PT_UPDATES); + + min *= sizeof(l1_pgentry_t); + length *= sizeof(l1_pgentry_t); + original = map_domain_mem(gmfn << PAGE_SHIFT); snapshot = map_domain_mem(smfn << PAGE_SHIFT); - memcpy(snapshot, original, PAGE_SIZE); + memcpy(snapshot + min, original + min, length); unmap_domain_mem(original); unmap_domain_mem(snapshot); @@ -2037,8 +2055,7 @@ static int resync_all(struct domain *d, u32 stype) unsigned long *guest, *shadow, *snapshot; int need_flush = 0, external = shadow_mode_external(d); int unshadow; - u32 min_max; - int min, max; + int changed; ASSERT(spin_is_locked(&d->arch.shadow_lock)); @@ -2063,31 +2080,49 @@ static int resync_all(struct domain *d, u32 stype) switch ( stype ) { case PGT_l1_shadow: - min_max = pfn_to_page(smfn)->tlbflush_timestamp; - min = SHADOW_MIN(min_max); - max = SHADOW_MAX(min_max); - for ( i = min; i <= max; i++ ) + { + u32 min_max_shadow = pfn_to_page(smfn)->tlbflush_timestamp; + int min_shadow = SHADOW_MIN(min_max_shadow); + int max_shadow = SHADOW_MAX(min_max_shadow); + + u32 min_max_snapshot = + pfn_to_page(entry->snapshot_mfn)->tlbflush_timestamp; + int min_snapshot = SHADOW_MIN(min_max_snapshot); + int max_snapshot = SHADOW_MAX(min_max_snapshot); + + changed = 0; + + for ( i = min_shadow; i <= max_shadow; i++ ) { - unsigned new_pte = guest[i]; - if ( new_pte != snapshot[i] ) + if ( (i < min_snapshot) || (i > max_snapshot) || + (guest[i] != snapshot[i]) ) { - need_flush |= validate_pte_change(d, new_pte, &shadow[i]); + need_flush |= validate_pte_change(d, guest[i], &shadow[i]); // can't update snapshots of linear page tables -- they // are used multiple times... // // snapshot[i] = new_pte; + + changed++; } } + perfc_incrc(resync_l1); + perfc_incr_histo(wpt_updates, changed, PT_UPDATES); + perfc_incr_histo(l1_entries_checked, max_shadow - min_shadow + 1, PT_UPDATES); break; + } case PGT_l2_shadow: - max = -1; + { + int max = -1; + + changed = 0; for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ ) { if ( !is_guest_l2_slot(i) && !external ) continue; - unsigned new_pde = guest[i]; + unsigned long new_pde = guest[i]; if ( new_pde != snapshot[i] ) { need_flush |= validate_pde_change(d, new_pde, &shadow[i]); @@ -2096,6 +2131,8 @@ static int resync_all(struct domain *d, u32 stype) // are used multiple times... // // snapshot[i] = new_pde; + + changed++; } if ( new_pde != 0 ) max = i; @@ -2109,14 +2146,18 @@ static int resync_all(struct domain *d, u32 stype) } if ( max == -1 ) unshadow = 1; + perfc_incrc(resync_l2); + perfc_incr_histo(shm_l2_updates, changed, PT_UPDATES); break; - default: + } + case PGT_hl2_shadow: + changed = 0; for ( i = 0; i < L2_PAGETABLE_ENTRIES; i++ ) { if ( !is_guest_l2_slot(i) && !external ) continue; - unsigned new_pde = guest[i]; + unsigned long new_pde = guest[i]; if ( new_pde != snapshot[i] ) { need_flush |= validate_hl2e_change(d, new_pde, &shadow[i]); @@ -2125,9 +2166,15 @@ static int resync_all(struct domain *d, u32 stype) // are used multiple times... // // snapshot[i] = new_pde; + + changed++; } } + perfc_incrc(resync_hl2); + perfc_incr_histo(shm_hl2_updates, changed, PT_UPDATES); break; + default: + BUG(); } unmap_domain_mem(shadow); @@ -2176,7 +2223,7 @@ void __shadow_sync_all(struct domain *d) unmap_domain_mem(ppte); } - // XXX mafetter: SMP perf bug. + // XXX mafetter: SMP // // With the current algorithm, we've gotta flush all the TLBs // before we can safely continue. I don't think we want to @@ -2186,7 +2233,10 @@ void __shadow_sync_all(struct domain *d) // (any path from a PTE that grants write access to an out-of-sync // page table page needs to be vcpu private). // - flush_tlb_all(); +#if 0 // this should be enabled for SMP guests... + flush_tlb_mask(((1 << smp_num_cpus) - 1) & ~(1 << smp_processor_id())); +#endif + need_flush = 1; // Second, resync all L1 pages, then L2 pages, etc... // @@ -2195,7 +2245,7 @@ void __shadow_sync_all(struct domain *d) need_flush |= resync_all(d, PGT_hl2_shadow); need_flush |= resync_all(d, PGT_l2_shadow); - if ( need_flush ) + if ( need_flush && !unlikely(shadow_mode_external(d)) ) local_flush_tlb(); free_out_of_sync_state(d); diff --git a/xen/include/asm-x86/shadow.h b/xen/include/asm-x86/shadow.h index 95e5090f18..ce6198aed9 100644 --- a/xen/include/asm-x86/shadow.h +++ b/xen/include/asm-x86/shadow.h @@ -52,9 +52,9 @@ #define shadow_lock(_d) do { ASSERT(!spin_is_locked(&(_d)->arch.shadow_lock)); spin_lock(&(_d)->arch.shadow_lock); } while (0) #define shadow_unlock(_d) spin_unlock(&(_d)->arch.shadow_lock) -#define SHADOW_ENCODE_MIN_MAX(_min, _max) (((L1_PAGETABLE_ENTRIES - (_max)) << 16) | (_min)) +#define SHADOW_ENCODE_MIN_MAX(_min, _max) ((((L1_PAGETABLE_ENTRIES - 1) - (_max)) << 16) | (_min)) #define SHADOW_MIN(_encoded) ((_encoded) & ((1u<<16) - 1)) -#define SHADOW_MAX(_encoded) (L1_PAGETABLE_ENTRIES - ((_encoded) >> 16)) +#define SHADOW_MAX(_encoded) ((L1_PAGETABLE_ENTRIES - 1) - ((_encoded) >> 16)) extern void shadow_mode_init(void); extern int shadow_mode_control(struct domain *p, dom0_shadow_control_t *sc); @@ -122,6 +122,8 @@ __shadow_sync_va(struct exec_domain *ed, unsigned long va) if ( d->arch.out_of_sync && __shadow_out_of_sync(ed, va) ) { + perfc_incrc(shadow_sync_va); + // XXX - could be smarter // __shadow_sync_all(ed->domain); @@ -1006,7 +1008,8 @@ static inline unsigned long __shadow_status( * Either returns PGT_none, or PGT_l{1,2,3,4}_page_table. */ static inline unsigned long -shadow_max_pgtable_type(struct domain *d, unsigned long gpfn) +shadow_max_pgtable_type(struct domain *d, unsigned long gpfn, + unsigned long *smfn) { struct shadow_status *x; unsigned long pttype = PGT_none, type; @@ -1014,6 +1017,8 @@ shadow_max_pgtable_type(struct domain *d, unsigned long gpfn) ASSERT(spin_is_locked(&d->arch.shadow_lock)); ASSERT(gpfn == (gpfn & PGT_mfn_mask)); + perfc_incrc(shadow_max_type); + x = hash_bucket(d, gpfn); while ( x && x->gpfn_and_flags ) @@ -1044,7 +1049,11 @@ shadow_max_pgtable_type(struct domain *d, unsigned long gpfn) } if ( type > pttype ) + { pttype = type; + if ( smfn ) + *smfn = x->smfn; + } } next: x = x->next; diff --git a/xen/include/xen/perfc_defn.h b/xen/include/xen/perfc_defn.h index b63fffd9d7..bcb3555206 100644 --- a/xen/include/xen/perfc_defn.h +++ b/xen/include/xen/perfc_defn.h @@ -2,6 +2,10 @@ #define PERFC_PT_UPDATES_BUCKET_SIZE 3 PERFCOUNTER_ARRAY( wpt_updates, "writable pt updates", PERFC_MAX_PT_UPDATES ) PERFCOUNTER_ARRAY( bpt_updates, "batched pt updates", PERFC_MAX_PT_UPDATES ) +PERFCOUNTER_ARRAY( l1_entries_checked, "l1 entries checked", PERFC_MAX_PT_UPDATES ) +PERFCOUNTER_ARRAY( shm_l2_updates, "shadow mode L2 pt updates", PERFC_MAX_PT_UPDATES ) +PERFCOUNTER_ARRAY( shm_hl2_updates, "shadow mode HL2 pt updates", PERFC_MAX_PT_UPDATES ) +PERFCOUNTER_ARRAY( snapshot_copies, "entries copied per snapshot", PERFC_MAX_PT_UPDATES ) PERFCOUNTER_ARRAY( hypercalls, "hypercalls", NR_hypercalls ) PERFCOUNTER_ARRAY( exceptions, "exceptions", 32 ) @@ -31,12 +35,21 @@ PERFCOUNTER_CPU( num_page_updates, "num_page_updates" ) PERFCOUNTER_CPU( calls_to_update_va, "calls_to_update_va_map" ) PERFCOUNTER_CPU( page_faults, "page faults" ) PERFCOUNTER_CPU( copy_user_faults, "copy_user faults" ) + +PERFCOUNTER_CPU(shadow_fault_calls, "calls to shadow_fault") +PERFCOUNTER_CPU(shadow_fault_bail_pde_not_present, "sf bailed due to pde not present") +PERFCOUNTER_CPU(shadow_fault_bail_pte_not_present, "sf bailed due to pte not present") +PERFCOUNTER_CPU(shadow_fault_bail_ro_mapping, "sf bailed due to a ro mapping") +PERFCOUNTER_CPU(shadow_fault_fixed, "sf fixed the pgfault") +PERFCOUNTER_CPU(write_fault_bail, "sf bailed due to write_fault") +PERFCOUNTER_CPU(read_fault_bail, "sf bailed due to read_fault") + PERFCOUNTER_CPU( map_domain_mem_count, "map_domain_mem count" ) -PERFCOUNTER_CPU( shadow_l2_table_count, "shadow_l2_table count" ) -PERFCOUNTER_CPU( shadow_l1_table_count, "shadow_l1_table count" ) -PERFCOUNTER_CPU( unshadow_table_count, "unshadow_table count" ) -PERFCOUNTER_CPU( shadow_fixup_count, "shadow_fixup count" ) +PERFCOUNTER_CPU( shadow_l2_table_count, "shadow_l2_table count" ) +PERFCOUNTER_CPU( shadow_l1_table_count, "shadow_l1_table count" ) +PERFCOUNTER_CPU( unshadow_table_count, "unshadow_table count" ) +PERFCOUNTER_CPU( shadow_fixup_count, "shadow_fixup count" ) PERFCOUNTER_CPU( shadow_update_va_fail1, "shadow_update_va_fail1" ) PERFCOUNTER_CPU( shadow_update_va_fail2, "shadow_update_va_fail2" ) @@ -62,8 +75,13 @@ PERFCOUNTER_CPU(shadow_status_shortcut, "fastpath miss on shadow cache") PERFCOUNTER_CPU(shadow_status_calls, "calls to ___shadow_status" ) PERFCOUNTER_CPU(shadow_status_miss, "missed shadow cache" ) PERFCOUNTER_CPU(shadow_status_hit_head, "hits on head of bucket" ) +PERFCOUNTER_CPU(shadow_max_type, "calls to shadow_max_type" ) PERFCOUNTER_CPU(shadow_sync_all, "calls to shadow_sync_all") +PERFCOUNTER_CPU(shadow_sync_va, "calls to shadow_sync_va") +PERFCOUNTER_CPU(resync_l1, "resync L1 page") +PERFCOUNTER_CPU(resync_l2, "resync L2 page") +PERFCOUNTER_CPU(resync_hl2, "resync HL2 page") PERFCOUNTER_CPU(shadow_make_snapshot, "snapshots created") PERFCOUNTER_CPU(shadow_mark_mfn_out_of_sync_calls, "calls to shadow_mk_out_of_sync") PERFCOUNTER_CPU(shadow_out_of_sync_calls, "calls to shadow_out_of_sync") @@ -71,11 +89,6 @@ PERFCOUNTER_CPU(extra_va_update_sync, "extra syncs for bug in chk_p PERFCOUNTER_CPU(snapshot_entry_matches_calls, "calls to ss_entry_matches") PERFCOUNTER_CPU(snapshot_entry_matches_true, "ss_entry_matches returns true") -PERFCOUNTER_CPU(shadow_fault_calls, "calls to shadow_fault") -PERFCOUNTER_CPU(shadow_fault_bail_pde_not_present, "sf bailed due to pde not present") -PERFCOUNTER_CPU(shadow_fault_bail_pte_not_present, "sf bailed due to pte not present") -PERFCOUNTER_CPU(shadow_fault_bail_ro_mapping, "sf bailed due to a ro mapping") -PERFCOUNTER_CPU(shadow_fault_fixed, "sf fixed the pgfault") PERFCOUNTER_CPU(validate_pte_calls, "calls to validate_pte_change") PERFCOUNTER_CPU(validate_pte_changes, "validate_pte makes changes") PERFCOUNTER_CPU(validate_pde_calls, "calls to validate_pde_change") @@ -92,6 +105,4 @@ PERFCOUNTER_CPU(remove_write_not_writable, "remove_write non-writable pa PERFCOUNTER_CPU(remove_write_fast_exit, "remove_write hit predicted entry") PERFCOUNTER_CPU(remove_write_predicted, "remove_write predict hit&exit") PERFCOUNTER_CPU(remove_write_bad_prediction, "remove_write bad prediction") -PERFCOUNTER_CPU(write_fault_bail, "sf bailed due to write_fault") -PERFCOUNTER_CPU(read_fault_bail, "sf bailed due to read_fault") -PERFCOUNTER_CPU(update_hl2e_invlpg, "update_hl2e calls invlpg") +PERFCOUNTER_CPU(update_hl2e_invlpg, "update_hl2e calls invlpg") -- 2.30.2